{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CIFAR10 example with ResNet56\n",
    "\n",
    "This example demonstrates the usage of `LRFinder` with a ResNet56 on the Cifar10 dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/davidtvs/datascience/pytorch/pytorch-lr-finder/env/lib/python3.6/site-packages/tqdm/autonotebook/__init__.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision.datasets import CIFAR10\n",
    "import cifar10_resnet as rc10\n",
    "\n",
    "try:\n",
    "    from torch_lr_finder import LRFinder\n",
    "except ImportError:\n",
    "    # Run from source\n",
    "    import sys\n",
    "    sys.path.insert(0, '..')\n",
    "    from torch_lr_finder import LRFinder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading CIFAR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar_pwd = \"../data\"\n",
    "batch_size= 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "trainset = CIFAR10(root=cifar_pwd, train=True, download=True, transform=transform)\n",
    "trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "\n",
    "testset = CIFAR10(root=cifar_pwd, train=False, download=True, transform=transform)\n",
    "testloader = DataLoader(testset, batch_size=batch_size * 2, shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = rc10.resnet56()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training loss (fastai)\n",
    "\n",
    "This learning rate test range follows the same procedure used by fastai. The model is trained for `num_iter` iterations while the learning rate is increased from its initial value specified by the optimizer algorithm to `end_lr`. The increase can be linear (`step_mode=\"linear\"`) or exponential (`step_mode=\"exp\"`); linear provides good results for small ranges while exponential is recommended for larger ranges."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16da02988f7543ce9ce6d1df4c2b8e15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping early, the loss has diverged\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-7, weight_decay=1e-2)\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device=\"cuda\")\n",
    "lr_finder.range_test(trainloader, end_lr=100, num_iter=100, step_mode=\"exp\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the loss in the loss vs. learning rate plot is the **training** loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEOCAYAAABmVAtTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XFX9//HXZ7K2SZqQJt2XdNUW6BqgpSAo4FdA9kVAKosCFRFQvv5ccP2qX3f8CqhQ2URZZQcBFS1C2UrShaYta9t0b9Omzdbs+fz+mCGGkrZpmzszN3k/H495cOfOmbmf05D55Jxzzznm7oiIiABEEh2AiIgkDyUFERFpp6QgIiLtlBRERKSdkoKIiLRTUhARkXZKCiIi0k5JQURE2ikpiIhIOyUFERFpl5roAPZVQUGBFxUVJToMEZFQKS0t3eruhXsrF7qkUFRURElJSaLDEBEJFTMr70o5dR+JiEg7JQUREWmnpCAiIu2UFEREpJ2SgoiItFNSEBGRdkoKIiIh8LdlmyjfVhf4dZQURESSXENzK1fdu5B7X1sT+LWUFEREklzZ+iqaW51pIw8K/FpKCiIiSW7hmu0ATBuhpCAi0uuVlm9nZP++FOZkBH4tJQURkSTm7pSW72B6HFoJoKQgIpLU1lbWs7W2MS7jCaCkICKS1ErXVAIwXUlBRERKy7eTnZHK+IE5cbleYEnBzIab2TwzW25my8zsmk7KfM3MFsceZWbWamb5QcUkIhI2peU7mDI8j5SIxeV6QbYUWoDr3H0iMAP4kplN7FjA3X/h7lPcfQrwTeDf7l4ZYEwiIqFR29jCW5uq4zaeAAEmBXff6O4LY8c1wApg6B7ecj5wX1DxiIiEzZK1O2jz+I0nQJzGFMysCJgKvLab1/sCnwIejkc8IiJhUFq+HTOYMjwvbtcMPCmYWTbRL/tr3b16N8VOAV7aXdeRmV1uZiVmVlJRURFUqCIiSaW0fDvjB+SQ2yctbtcMNCmYWRrRhHCPuz+yh6LnsYeuI3ef6+7F7l5cWFjY3WGKiCSdtjZn4ZrtcR1PgGDvPjLgdmCFu9+wh3K5wDHA40HFIiISNu9W1FLT0BLX8QSA1AA/exYwG1hqZotj574FjABw91ti584A/u7uwS8ULiISEgvLo4vg9Zik4O7zgb3eWOvudwF3BRWHiEgYlZZvJz8rnaL+feN6Xc1oFhFJQqVrtjNtRB7Rnvj4UVIQEUkylXVNrKyoi/sgMygpiIgknWfLNgHEbbnsjoIcaBYRkX3g7sx9YSU/ffZNJg/PY6qSgohI79TU0sb1jy7lL6XrOPnQwfzynMmkp8a/M0dJQUQkwSrrmpjz51IWrKrk6k+M5drjxxOJ06qou1JSEBFJIHfnsrtLWLq+it+cN4XTpuxp3dDgaaBZRCSBFqyqpLR8O9/59MSEJwRQUhARSag/vLiS/Kx0zpk+LNGhAEoKIiIJ8+6WWp5bsYXZM0aSmZaS6HAAJQURkYS5ff4q0lMjzJ45MtGhtFNSEBFJgK21jTy8cB1nTRtGQXZGosNpp6QgIpIAd79STlNLG58/alSiQ/kAJQURkTirb2rlz6+Wc/yEAYwdkJ3ocD5ASUFEJM4eXriOyromvnD06ESH8iFKCiIicdTW5tw+fxWThuVyxKj8RIfzIUoKIiJxtHxjNau21vG5mUVx3yuhK5QURETiaFtdE0Dcd1TrKiUFEZE4qqpvBiC3T1qCI+mckoKISBwpKYiISLvqWFLop6QgIiJV9c1kpEaSZq2jXSkpiIjEUdXO5qTtOgIlBRGRuKqqV1IQEZEYJQUREWmnpCAiIu2UFEREpF11fXPS3o4KSgoiInHT2ubUNLaopSAiIv+ZuNYrk4KZDTezeWa23MyWmdk1uyl3rJktjpX5d1DxiIgkWrIvcQGQGuBntwDXuftCM8sBSs3sH+6+/P0CZpYH/A74lLuvMbMBAcYjIpJQYUgKgbUU3H2juy+MHdcAK4ChuxS7AHjE3dfEym0JKh4RkUSrboglhb69MCl0ZGZFwFTgtV1eGg8cZGbPm1mpmX0uHvGIiCRCGFoKQXYfAWBm2cDDwLXuXt3J9acDxwF9gFfM7FV3f3uXz7gcuBxgxIgRQYcsIhKIMCSFQFsKZpZGNCHc4+6PdFJkHfA3d69z963AC8DkXQu5+1x3L3b34sLCwiBDFhEJTK9OChbdfPR2YIW737CbYo8DR5lZqpn1BY4gOvYgItLjVNU3k57Ey2ZDsN1Hs4DZwFIzWxw79y1gBIC73+LuK8zsWeANoA24zd3LAoxJRCRhquub6ZeZvK0ECDApuPt8wLpQ7hfAL4KKQ0QkWUTXPQp8KPeAaEaziEicJPtieKCkICISN0oKIiLSTklBRETaJfv+zKCkICISF20hWDYblBREROKipqEFd5J6gx1QUhARiYswzGYGJQURkbhQUhARkXZKCiIi0q49KSTxXgqgpCAiEhdqKYiISDslBRERaVdV30xaitEniZfNhjjsvCY9j7uzY2cz1Q3R2Zk5mWmkRPa6IK5Ir/b+EhfRrWaSl5KC7NHGqnpKVm+ntHw7q7bWsX5HPRt21LOzqbW9jBnkZKSS1zedYQf1YVRBVvtj0rA8CnMyElgDkeRQXd+c9BPXQEmhx2trc2oaWjq946GxpZVXV1byrxWb2VjVQFpKhNQUIy0lQkNzK4vW7GD9jnoA+qSlMGZAFmMKs/jYuEKG5GWS2yeN6oYWquqbqa5vprKuiTWVO3nqjY3t/afpKRHOmj6MOceMZmT/rLjWXSSZhGExPFBS6NFWba3jmvsX8ca6Kgqy0xk/MIePDMphaF4fSsu388LbFdQ1tZKZFmFkfhYtbW20tDktrU4kAlOG5/H5o0ZxWFE+EwbnkJrS9SGo7XVNrNxay6OL1vNgyToeeH0NJ08awhePGcPEIf0CrLVIcqqqb6Z/dnqiw9grJYUksLW2kdvnr+KdzbWMKcxiTGE2YwZkM7Ywe7/uaXZ3Hl64nu8+XkZ6aoRrjx/Hhh31vLWphvsXrKW+uZVB/TI5bepQjp8wgCPHFHT7nrEHZaUzPSuf6SPzufq4cdw+fxX3vLqGJ5dsYPzAbE6YOJDjJwxk8rA8IhqPkF6gqr6Z0YXJ31ruVUmhrc3ZVtfEpqoGNlTVs6mqgc3VDWypaaQi9qhpbCYjNYXMtAiZqSn0SU9h3IAcpozIY8qwPIbn98HMaGxpZc22nbxXUcuayp3UNbbS0NJKY3MbDc2tHJSVzviB2YwbkMPYAdmdfuluqmrg1hfe474Fa2hqaWNUQRYvvFNBU0tbe5kJg/vxsfEFHDO+kOKR+aSn7vmv9ZqGZr79WBmPL97AEaPy+b/zpjA4t88H/g221jZSmJMRtwGvATmZfPPECVx5zFgeWbSOfyzfzC3/Xslv573HgJwMrjl+HJ89YmRcYhFJFHUfJZmnl27k2vsX09Ta9oHzqRGjIDuDAf0yGJKXSU5mDo0trTTEvtyr6pu557Vy7nhpFQD9s9LJzkxlbeVO2vyD10hPjZCZGiEjLYXtdU20xApEDAbn9iGvbxr9MtPo1yeViBn/XLGFVnfOmDqUK48dw+jCbFrbnHXbd/LullpWbKxm/rtbuf3FVdz675X0TU9hUL9M6ptbaWiOxtjc2kbEjEgEUsyi3T9tznUnjOfKj4/90F1BkYgxoF9mcP/Qe5DbN41LZo3iklmj2LGziXlvbeH+BWu5/tEy2hxmz1BikJ6prc3b79ZLdr0mKYwpzOaSWUUMzs1kcF4fhuT2YVBuJv2z0vfafdHc2sZbm2pYvHYHi9fuoL6pldMmD2F0YTZjCrMZWdCX7PTUD3xOc2sbq7fW8fbmWt7eXMOayp1U10dv41y9dSe1jS2cNX0YVx47huH5fdvflxIxRvbPYmT/LI6bMJCrPjGO2sYWXnlvGy++U8H2nc1kpkbITIu2ZtJSIrQ5tLnT2ua4w8mTBjN95EGB/Vt2h7y+6ZwxdRgnHzqEK+8p5TuPlZGREuHcw4YnOjSRblfTGF02OwxJwdx976WSSHFxsZeUlCQ6DOlGjS2tXHZ3KS++U8Gvz53C6VOHJjokkW61tnInR/98Hj8/exLnFifmDx8zK3X34r2V04xmSbiM1BTmzp7OjFH9+eqDi3lyyYZEhyTSrcKyxAUoKUiSyExL4faLi5k+8iC+fN8iTr7xRe58aRWVdU2JDk3kgCkpiOyHvump3H3pEXz/lIlEzPjBk8s54n+f44o/lfDqym2JDk9kv4UpKfSagWYJhz7pKVw8axQXzxrFm5uqebh0HY8sXM/flm3m8FH5XHvcOGaO6Z/068eIdBSmpKCWgiStjw7qx/UnT+Slb3yC750ykfJtdVxw22uce+srzHtrC2273hMskqTClBTUUpCkl5mWwiWzRnH+4SN4sGQtv3/+PS6583VGFWRx4YyRnD19WCh+2aT3qqpvJjVi9E1P7mWzQS0FCZHMtBQ+N7OIf3/t4/zmvCnkZ6Xzw6eWM+N//8k3H3mDd7fUJDpEkU6FZdlsCLClYGbDgbuBgYADc939N7uUORZ4HFgVO/WIu/9PUDFJz5CeGuG0KUM5bcpQytZX8edXy3l00XruW7CW4ycMZM4xoykuyk90mCLtwrLEBQTbfdQCXOfuC80sByg1s3+4+/Jdyr3o7p8OMA7pwQ4ZmstPz5rE//vUR/njy6u5+5XVnH3LZqaPPIirjxvHMeMLEx2iSGj2UoAAu4/cfaO7L4wd1wArAE1VlUDkZ6XzlRPG89I3PsEPTj2YzdUNXHTHAi65cwHvVdQmOjzp5apD1FKIy5iCmRUBU4HXOnl5ppktMbNnzOzgeMQjPVff9FQuOrKIf113LNefNIGS1dv5r1+/wA+fWt5+B4hIvIWp+yjwpGBm2cDDwLXuXr3LywuBke4+GbgJeGw3n3G5mZWYWUlFRUWwAUuPkJ4a4bKPjWbe147lnOJh3PHSKo771fPMf2drokOTXqjHJQUzG2NmGbHjY83sajPL68L70ogmhHvc/ZFdX3f3anevjR0/DaSZWUEn5ea6e7G7FxcWqo9Yuq4gO4OfnDmJJ686ivysdGbf8Ro3/+sdzXGQuHF3qhtaelZSIPrF3mpmY4G5wHDg3j29waL3Xt0OrHD3G3ZTZlCsHGZ2eCwerWcg3e6Qobk8euUsTpk0hF/+/W0u/1OJupMkLmobW2ht8x6XFNrcvQU4A7jJ3b8GDN7Le2YBs4FPmNni2OMkM5tjZnNiZc4GysxsCXAjcJ6HbS1vCY2sjFR+c94UfnDqwTz/VgWn3DSfsvVViQ5LergwzWaGrt+S2mxm5wMXAafEzu2xhu4+H9jjTA13vxm4uYsxiBwwM+OiI4s4ZGguX7pnIaf/9iWuPX4cc44ZQ2qK5nJK93s/KfS0W1IvAWYCP3b3VWY2CvhTcGGJBGv6yIN49tqjOfHQwfzy729z7q2vsHprXaLDkh4obC2FLiUFd1/u7le7+31mdhCQ4+4/Czg2kUDl9U3npvOn8pvzpvDullpOuvFFHnh9TaLDkh6muicmBTN73sz6mVk+0dtI/2BmnQ4ei4TNaVOG8revfIypI/L4+sNL+fFfl+vuJOk27S2Fvj0oKQC5sTkGZwJ3u/sRwPHBhSUSX4Nz+3D3pUdw8ZFF/OHFVVx130IamlsTHZb0AD2y+whINbPBwLnAUwHGI5IwKRHje6dM5NsnT+CZsk1c8IdXtR2oHLCq+mZSIkZWCJbNhq4nhf8B/ga85+6vm9lo4J3gwhJJDDPjC0eP5rcXTKNsQzVn/u4l1lbuTHRYEmJhWjYbuj7Q/Bd3n+TuX4w9X+nuZwUbmkjinHToYO677Ah21DfrziQ5INX1LfTLDM9+Zl0daB5mZo+a2ZbY42EzGxZ0cCKJNH1kPvd+YQaNLW2ce+srvLtFq63KvgvTukfQ9e6jO4EngCGxx5OxcyI92sQh/bj/8hm0OZw39xXe2qTd3WTfVDeEZy8F6HpSKHT3O929Jfa4C9DKdNIrjB+Yw/2XzyBixvl/eJVlG7Q0hnRdVYg22IGuJ4VtZnahmaXEHheiheukFxk7IJsHrphJRmqEc255haeXbkx0SBIS0TGFnpcULiV6O+omYCPRhewuDigmkaQ0qiCLR6+cxUcG5XDlPQv5yTMraGltS3RYkuSi3Uc9bKDZ3cvd/VR3L3T3Ae5+OqC7j6TXGZSbyf2Xz+CzR4zg1n+v5KI7F2gug+xWQ3MrTS1tPXKguTNf7bYoREIkIzWFH59xKD8/exKvr97OKTfNZ912zWWQD3t/3aOe2H3UmXDMxBAJyLnFw3lozkxqGpq59K7XqW7Qpj3yQWFb4gIOLCloxTDp9SYNy+OW2dNZtbWOK/+8kGaNMUgH7/+h0GPuPjKzGjOr7uRRQ3S+gkivd+SYAn5y5iTmv7uV6x9dijYPlPdV17cAhGpG8x4jdfeceAUiEmZnTx/Gmsqd3PjPdxjZP4svfXxsokOSJBDG7qPwpC+RJPeV48extnInv/jbWwzP78upk9WY7u16XPeRiHSdmfHTsw7l8KJ8vvaXJSxdp5nPvV3Vzt5195GI7CIjNYXfXTiNguwMLv9TCVtqGhIdkiRQdUMzfdJSSE8Nz1dteCIVCYmC7Azmfm4623c28cU/L6SxRTu49VbV9S2hms0MSgoigTh4SC6/PGcypeXb+e5jy3RHUi8VtmWzQQPNIoH59KQhvLmxhpvnvcvEIf246MiiRIckcVbd0Byq8QRQS0EkUF89YTzHTxjI/zy1nNLy7YkOR+IsbMtmg5KCSKAiEeNX505mSF4mV9+3iB07tXheb1LdEL7uIyUFkYDl9knj5vOnsaWmgf/+yxsaX+hFwrY/MygpiMTF5OF5fOPECTy3YjN3vLQ60eFIHLS1uVoKIrJ7l84q4oSJA/npMytYsnZHosORgNU2teAertnMoKQgEjdmxi/OnsSAnEy+dO/C9nVxpGcK42xmCDApmNlwM5tnZsvNbJmZXbOHsoeZWYuZnR1UPCLJIK9vOjeeP5VNVQ1c9+Bi2to0vtBThXHdIwi2pdACXOfuE4EZwJfMbOKuhcwsBfgZ8PcAYxFJGtNHHsS3T57Acyu2cPO8dxMdjgSkfdlszWiOcveN7r4wdlwDrACGdlL0y8DDwJagYhFJNhcdWcQZU4fy6+feZt6b+l+/J6oK4VacEKcxBTMrAqYCr+1yfihwBvD7eMQhkizMjP8941AmDOrHNfcvonxbXaJDkm72fveR7j7ahZllE20JXOvu1bu8/H/A1919j3sYmtnlZlZiZiUVFRVBhSoSV33SU7h19nTMjCv+VMrOppZEhyTdqLpeYwofYmZpRBPCPe7+SCdFioH7zWw1cDbwOzM7fddC7j7X3YvdvbiwsDDIkEXianh+X248fypvba7hW49oK8+epLq+GTPIydCYAgBmZsDtwAp3v6GzMu4+yt2L3L0IeAi40t0fCyomkWR0zPhCvnL8eB5bvIFHF61PdDjSTaobWsjJSCUSsUSHsk+CbCnMAmYDnzCzxbHHSWY2x8zmBHhdkdD50sfHcnhRPt95rEzjCz1EGBfDgwCXznb3+UCXU6S7XxxULCLJLiVi/Pq8KZz4fy9wzf2L+cucmaSlaG5pmFWHcC8F0IxmkaQxNK8P/3vmoSxeu4PfPPdOosORAxTGvRRASUEkqXx60hDOmT6M3z7/Lq+u3JbocOQARLuPwjXIDEoKIknn+6ceTFH/LL7ywGLWVu5MdDiyn6rrW9R9JCIHLisjlRvPm0ptYwsn3fgiz5ZtSnRIsh+q6tV9JCLd5NBhufz1y0czqiCLOX8u5QdPLqOpZY9zPCWJNLW0Ud/cqpaCiHSfEf378pc5M7lkVhF3vrSas295mXXb1Z0UBjUhXSEVlBREklpGagrfO+VgbrlwOqu21vGFP5bQ2NKa6LBkL9oXw9NAs4gE4VOHDOLG86by5qYabvj724kOR/aiuiG6jpW6j0QkMB//6AAuOGIEc19cqdtVk1xYl80GJQWRULn+pAmMzO/LdQ8uae+3luTz/gqpaimISKCyMlK54TNT2FhVzw+eXJ7ocGQ3wroVJygpiITOtBEHcdXHx/JQ6TqeLduY6HCkE+o+EpG4+vJx4zh0aC7feGQpyzfsuneVJFp1fQvpKREy08L3FRu+iEWEtJQIN18wlb5pKZw39xUWr92R6JCkg/fXPYpuKxMuSgoiITWyfxYPXDGTvL7pXHjbayxYVZnokCQmrCukgpKCSKgNz+/Lg1fMZEC/DD53x2vMf2drokMSoncfhXGQGZQUREJvUG4mD1w+k6L+WVz6x9f52zItoJdoSgoiklCFORncf/kMJg7ux5w/l3L3K6sTHVKvVt0QzmWzQUlBpMfI65vOfZfN4LiPDuC7jy/jJ8+soK3NEx1WrxRdNjt86x6BkoJIj9InPYVbZxdz4YwR3PrvlVzzwGItoBdn7h7q7qNwpjIR2a2UiPHD0w5haF5ffvbsm9Q2NHPHxYeF8vbIMKpvbqWlzdV9JCLJw8z44rFj+M6nJzLvrQoeX7wh0SH1GmGezQxKCiI92sVHFjF5eB4/+uuK9i8rCVZ1fXTZ7DDupQBKCiI9WkrE+NFph1BZ18gNf38r0eH0Cu8vhqfuIxFJSocOy2X2jJH86dVyytZXJTqcHq9qp7qPRCTJffWTHyE/K4PrHyvTbaoBU0tBRJJebp80vn3yBJas3cH9r69NdDg92n/2Z1ZSEJEkdtqUIcwYnc/Pnn2TrbWNiQ6nx3p/oDlHk9dEJJmZGT86/RDqm1r57uNluKsbKQjVDc1kpaeQlhLOr9fAojaz4WY2z8yWm9kyM7umkzKnmdkbZrbYzErM7Kig4hERGDsgh2tPGMfTSzfx1BvatS0IVSGezQzBthRagOvcfSIwA/iSmU3cpcw/gcnuPgW4FLgtwHhEBLj86NFMHp7Hdx4vY0tNQ6LD6XGq65tDO8gMASYFd9/o7gtjxzXACmDoLmVq/T9t2CxA7VmRgKWmRPjVOZPY2dTK9Y+qG6m7RRfDU1LYIzMrAqYCr3Xy2hlm9ibwV6KtBREJ2NgBOXztkx/hH8s389ji9YkOp0epbmgJ7WxmiENSMLNs4GHgWnf/0A7j7v6ou38UOB344W4+4/LYmENJRUVFsAGL9BKXHjWK6SMP4nuPL2NztbqRukuYV0iFgJOCmaURTQj3uPsjeyrr7i8Ao82soJPX5rp7sbsXFxYWBhStSO+SEjF+cfYkmlrbuO7BJbRqUlu3qFb3Uecsuk7v7cAKd79hN2XGxsphZtOADGBbUDGJyAeNLszmB6cezPx3t3LDP7Q20oFqaW2jprEl1C2FIDu+ZgGzgaVmtjh27lvACAB3vwU4C/icmTUD9cBnXKNeInH1mcNGsGjNDn477z0mD8vjkwcPSnRIobVqax0AI/P7JjiS/RdYUnD3+cAed/Vw958BPwsqBhHpmu+fejDLN1Zz3YNLePyqbEYXZic6pFBaGltw8NBhuQmOZP+Fc8qdiHSrzLQUfn/hdNJSI1zxp1LqGlsSHVIola2vJjMtwuiCrESHst+UFEQEgKF5fbjp/Km8V1HLdQ8uaV/tU7qubEMVEwf3IzWkS1yAkoKIdDBrbAHfPHECzy7bxKyf/IufPLOCLbpdtUva2pzlG6o5ZGh4u45ASUFEdnHZx0bz5FVHccxHCvnDCys56mfz+PpDb2guw16s3lZHbWNL6JNCeKfdiUhgDh2Wy80XTKN8Wx23vbiKB0vWsmB1JQ9cMYMBOZmJDi8plW2Izs09ZEi4k4JaCiKyWyP7Z/HD0w/hni8cwaaqBmbftoDtdU2JDispla2vIj01wriB4b5zS0lBRPaquCif2y4qZtW2Oi66cwE1GoT+kLL1VUwYlBPafRTeF+7oRSRuZo0t4HcXTGP5hmo+f1cJ9U2tiQ4pabg7ZeurODjk4wmgpCAi++D4iQP59WemUFJeyUV3LOC9itpEh5QU1lbWU93QEvrxBFBSEJF9dMrkIfz6M1NYsbGa//r1C/zoqeW9fk5D2YbYTGa1FESkNzptylD+9d/Hcvb0Ydz+0io+/ovnufe1NTS29M4upbL1VaRGjPGDwj3IDEoKIrKfCnMy+OlZk3jyqqMYXZjFtx5dymE/eo5vPPwGr7y3jbZetBT30vVVjB+YQ0ZqSqJDOWCapyAiB+SQobk8eMVMXnhnK48tWs8TSzZw/+trGdQvk5lj+jMoN5NB/TIZlJvJ6IIsxg3MSXTI3crdWbahmhMmDEx0KN1CSUFEDpiZccz4Qo4ZX0h9UyvPrdjME0s2sGBVJVtqGmhu/U+r4eIji/jmSR/tEX9VA2yoaqCyrolDhvZLdCjdQklBRLpVn/QUTpk8hFMmDwGiawJtq2tic3UDD5Wu466XV1NSXsnN50+jKMSrib6vLLZcdk+4HRU0piAiAYtEjMKcDA4Zmsv3Tz2YubOns7aynk/fNJ8nlmxIdHgHbNn6KlIixsTBPaOloKQgInH1yYMH8fQ1RzN+YDZX37eIa+9fREVNY6LD2m9L11cxtjCbzLSe0R2mpCAicTc0rw8PXDGTq48bx1+XbuS4Xz3PPa+Vh/KOpbIesFx2R0oKIpIQaSkRvnrCeJ655mNMHNKP6x8t48zfv8zSdVWJDq3LtlQ3UFHT2GMGmUFJQUQSbOyAbO67bAa//sxk1lbu5JSb53P271/mkYXraGhO7slwb8QSWE9qKZh7uJprxcXFXlJSkugwRCQAVTubeaBkDfctWMuqrXX0y0zlzGnDOHxUPuMHZjOyf1bSrEL67pYaPnf7AuqaWnnlm5+gb3py38xpZqXuXrzXckoKIpJs3J1XV1Zy34I1PFu2iabWNgDSUoxRBVlMHX4QJx46iFljCxKSJErLK7n0rhLSUiLcdclhoWgpdDUpJHdqE5FeycyYOaY/M8f0p76plfcqanl7cw1vb47+969LN/JAyVpy+6TxyYkDOfHQQRQX5dMvMy3w2J5bvpmr7lvIoH6Z3H3pEYzo3zfwa8aTkoKIJLU+6SkcMjT3A3+NNzS38uI7W3l66UaeLdvEX0rXYQafg9c9AAAI80lEQVRjC7OZOiKPKcMPAqC8so7yrTspr9xJQ3MrhxUdxKyxBRw5poDCnIx9iqOtzblnwRq+/8QyJg7ux52XHEZB9r59Rhio+0hEQq2xpZXXV21n0ZrtLFq7g0VrtrN9Z3Qp7/SUCMPy+1DUP4uIGQtWbaO6oQWAjw7K4Zzi4Zx/+PC9jge8sW4H339iGQvX7ODocQX8/sLpZGeE629qjSmISK/k7qyp3ElqSoRB/TJJiVj7a61t0R3SXnpvK/9csYXS8u3kZ6Xz+aNGMXvmyA91P22tbeSXf3uLB0rW0j8rg69/6iOcNW0YkQ6fGRZKCiIie1GyupKb573L829VkJORyqyxBdQ1tVDT0EJNQzMbdjTQ3NrGpUeN4sufGEtOHMYsgqKkICLSRWXrq/j98+/x5qZqcjLTyMlMpV9mGv2z0/nczCLGDgj/5jm6+0hEpIsOGZrLbz87LdFhJIXkmAUiIiJJIbCkYGbDzWyemS03s2Vmdk0nZT5rZm+Y2VIze9nMJgcVj4iI7F2Q3UctwHXuvtDMcoBSM/uHuy/vUGYVcIy7bzezE4G5wBEBxiQiInsQWFJw943AxthxjZmtAIYCyzuUebnDW14FhgUVj4iI7F1cxhTMrAiYCry2h2KfB57ZzfsvN7MSMyupqKjo/gBFRASIQ1Iws2zgYeBad6/eTZmPE00KX+/sdXef6+7F7l5cWFgYXLAiIr1coLekmlka0YRwj7s/spsyk4DbgBPdfVuQ8YiIyJ4FefeRAbcDK9z9ht2UGQE8Asx297eDikVERLomsBnNZnYU8CKwFGiLnf4WMALA3W8xs9uAs4Dy2Oste5txZ2YVwA6g4559uR2e7+24ANi6f7X6wOfty+udnd/1nOrQdarDns+pDl3Xm+ow0t333v/u7qF7AHN393xvx0BJd123q693dl51UB1UB9UhkXXY3SOsM5qf3MPzrhx313W7+npn51WH/ac67Pmc6tB1vbkOnQrdgngHysxKvAuLQiUz1SE5qA7JQXXoXmFtKRyIuYkOoBuoDslBdUgOqkM36nUtBRER2b3e2FIQEZHdUFIQEZF2SgoiItJOSaEDM4uY2Y/N7CYzuyjR8ewPMzvWzF40s1vM7NhEx7O/zCwrtgjipxMdy/4wswmxn8FDZvbFRMezP8zsdDP7g5k9YGafTHQ8+8PMRpvZ7Wb2UKJj2Rex////GPv3/2w8r91jkoKZ3WFmW8ysbJfznzKzt8zsXTP7xl4+5jSiy3c3A+uCinV3uqkODtQCmYS3DhBdHPHBYKLcs+6og7uvcPc5wLnArCDj7Uw31eExd78MmAN8Jsh4O9NNdVjp7p8PNtKu2cf6nAk8FPv3PzWuge7vLLpkewAfA6YBZR3OpQDvAaOBdGAJMBE4FHhql8cA4BvAFbH3PhTSOkRi7xtIdCHCMNbhBOA84GLg02GsQ+w9pxJdDv6CsNYh9r5fAdNCXoe4/z4fYH2+CUyJlbk3nnEGukpqPLn7C7F9Gzo6HHjX3VcCmNn9wGnu/hPgQ90SZrYOaIo9bQ0u2s51Rx062A5kBBHnnnTTz+FYIIvoL0e9mT3t7m27lgtKd/0c3P0J4Akz+ytwb3ARd3rt7vg5GPBT4Bl3XxhsxB/Wzb8PCbcv9SHayh8GLCbOPTo9JinsxlBgbYfn69jzdp+PADeZ2dHAC0EGtg/2qQ5mdibwX0AecHOwoXXZPtXB3a8HMLOLga3xTAh7sK8/h2OJdgFkAE8HGlnX7evvw5eB44FcMxvr7rcEGVwX7evPoT/wY2CqmX0zljySye7qcyNws5mdTPcshdFlPT0p7BN330l0s5/Q8ui+FZ3uXRE27n5XomPYX+7+PPB8gsM4IO5+I9Evp9Dy6B4tcxIdx75y9zrgkkRcu8cMNO/GemB4h+fDYufCRHVIDqpDcugJdego6erT05PC68A4MxtlZulEBy+fSHBM+0p1SA6qQ3LoCXXoKPnqk+gR+W4c2b8P2Mh/bif9fOz8ScDbREf4r090nKqD6qA69J46hLE+WhBPRETa9fTuIxER2QdKCiIi0k5JQURE2ikpiIhIOyUFERFpp6QgIiLtlBSkxzCz2jhf7zYzmxjna15rZn3jeU3pXTRPQXoMM6t19+xu/LxUd2/prs/r4jWN6O9lp4sAmtlqoNjdt8YzLuk91FKQHs3MCs3sYTN7PfaYFTt/uJm9YmaLzOxlM/tI7PzFZvaEmf0L+KdFd7J73qI7qL1pZvfEvriJnS+OHddadNe+JWb2qpkNjJ0fE3u+1Mx+1FlrxsyKYpus3A2UAcPN7PcW3XlumZn9IFbuamAIMM/M5sXOfTJWj4Vm9hcz67akKL1UoqdU66FHdz2A2k7O3QscFTseAayIHfcDUmPHxwMPx44vJroEQX7s+bFAFdGFyiLAKx0+73mif7VDdMe7U2LHPwe+HTt+Cjg/djxnNzEWAW3AjA7n3r9+Suw6k2LPVwMFseMCoku8Z8Wefx34bqJ/DnqE+6Gls6WnOx6YGPvjHqBf7K/pXOCPZjaO6Bd6Wof3/MPdKzs8X+Du6wDMbDHRL/H5u1yniWgCACglunscwEzg9NjxvcAvdxNnubu/2uH5uWZ2OdHl7QcT3XDojV3eMyN2/qVY/dKJJi2R/aakID1dhOhf4A0dT5rZzcA8dz8jthvW8x1ertvlMxo7HLfS+e9Ns7v7XsrsSfs1zWwU8N/AYe6+3czuIrrn9q6MaAI7fx+vJbJbGlOQnu7vRHcQA8DMpsQOc/nPuvUXB3j9V4GzYsfndfE9/YgmiarY2MSJHV6rAXI6fPYsMxsLYGZZZjb+wEOW3kxJQXqSvma2rsPjq8DVQLGZvWFmy/nPLlw/B35iZosItsV8LfBVM3sDGEt0fGKP3H0JsAh4k2iX00sdXp4LPGtm89y9gmhCuy/2+a8AH+3e8KW30S2pIgGKzSmod3c3s/OIDjqflui4RHZHYwoiwZpOdAN2A3YAlyY4HpE9UktBRETaaUxBRETaKSmIiEg7JQUREWmnpCAiIu2UFEREpJ2SgoiItPv/KXwLxqqOJXwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_finder.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To restore the model and optimizer to their initial state use the `reset()` method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_finder.reset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation loss (Leslie N. Smith)\n",
    "\n",
    "If a dataloader is passed to `LRFinder.range_test()` through the `val_loader` parameter the model is evaluated on that dataset after each iteration. The evaluation loss is more sensitive to instability therefore it provides a more precise view of when the divergence occurs. The disadvantage is that it takes significantly longer to run.\n",
    "\n",
    "This version of the learning rate range test is described in [Cyclical Learning Rates for Training Neural Networks by Leslie N. Smith](https://arxiv.org/abs/1506.01186)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "772150720509477b93829bfa89d1703d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping early, the loss has diverged\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    }
   ],
   "source": [
    "lr_finder.range_test(trainloader, val_loader=testloader, end_lr=100, num_iter=100, step_mode=\"exp\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the loss in the loss vs. learning rate plot is the **evaluation** loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEOCAYAAABIESrBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VPW9//HXJxtZCFsIawggqwioEKiKKFattrXFrSK1rVZbaustte2ttndrf22teuvVXm2rxaWidRetu5WrIqKAbIoIKDsEWcK+ZLJ/fn/MAVMETCaZmTPJ+/l4zCNnvnNmzudrcD75Luf7NXdHREQkFmnJDkBERFKXkoiIiMRMSURERGKmJCIiIjFTEhERkZgpiYiISMyUREREJGZKIiIiEjMlERERiZmSiIiIxCwjXh9sZvcB5wFb3X1ovfIfAtcAtcAL7n5dUP4L4KqgfLK7/+OzrtG5c2fv06dPHKIXEWm5FixYsM3dC5vjs+KWRID7gT8CDxwoMLMzgPHA8e5eaWZdgvIhwKXAcUAP4P/MbKC71x7tAn369GH+/PlxCl9EpGUys3XN9Vlx685y95nAjkOKvw/c5O6VwTlbg/LxwKPuXunua4CVwOh4xSYiIs0j0WMiA4GxZjbXzN4ws1FBeU9gQ73zSoMyEREJsXh2Zx3pep2Ak4BRwONmdkxjPsDMJgGTAIqLi5s9QBERabhEt0RKgac86h2gDugMbAR61TuvKCj7FHef4u4l7l5SWNgs40IiIhKjRCeRvwNnAJjZQCAL2AY8C1xqZm3MrC8wAHgnwbGJiEgjxXOK7yPAOKCzmZUCvwTuA+4zsyVAFXC5R7dW/MDMHgeWAjXANZ81M0tERJLPUnl73JKSEtcUXxFpbaYv3UKfglwGdM2P6f1mtsDdS5ojFt2xLiKSYq55aCHTFh522DjhlERERFJITW0dVbV15GalJzsUQElERCSlRKqjw8U5mUoiIiLSSAeSSLZaIiIi0lgVVXUA5KolIiIijVVeXQNAjloiIiLSWJEqjYmIiEiMDo6JKImIiEhjHWiJaIqviIg02sEpvkoiIiLSWBoTERGRmFWoJSIiIrEqV0tERERipdlZIiISs0h1LVkZaaSnWbJDAZRERERSSqSqNjTTe0FJREQkpUSqakMzHgJKIiIiKSVSrSQiIiIxqqiuDc30XohjEjGz+8xsq5ktOcxrPzUzN7POwXMzs9vNbKWZLTazEfGKS0QklZW3ou6s+4FzDy00s17AF4D19Yq/CAwIHpOAO+MYl4hIyoq0lpaIu88EdhzmpduA6wCvVzYeeMCj5gAdzKx7vGITEUlVkara0NwjAgkeEzGz8cBGd3/vkJd6AhvqPS8NykREpJ5Idbim+GYk6kJmlgv8G9GurKZ8ziSiXV4UFxc3Q2QiIqmjNU/x7Qf0Bd4zs7VAEbDQzLoBG4Fe9c4tCso+xd2nuHuJu5cUFhbGOWQRkXCJVLfS7ix3f9/du7h7H3fvQ7TLaoS7bwaeBb4VzNI6Cdjt7psSFZuISKqoCFl3Vjyn+D4CzAYGmVmpmV11lNNfBFYDK4G7gR/EKy4RkVRVXVtHda2HqjsrbmMi7j7xM17vU+/YgWviFYuISEsQtl0NQXesi4ikjIqqcC0DD0oiIiIp48CGVK1iTERERJrXwe4stURERKSxDu5qqJaIiIg01oExkVy1REREpLEOjIlodpaIiDSaxkRERCRmB8dElERERKSxIpriKyIisdId6yIiErMDLZHsDCURERFppIrqWrIz00hLs2SHcpCSiIhIiigP2YZUoCQiIpIyItVKIiIiEqNIdW2oljwBJRERkZQRqQrXroagJCIikjIiGhMREZFYRaprQ3W3OiiJiIikjFbVnWVm95nZVjNbUq/s92a23MwWm9nTZtah3mu/MLOVZvahmZ0Tr7hERFJVa5uddT9w7iFl04Gh7j4c+Aj4BYCZDQEuBY4L3vNnMwvXfykRkSSLVNeGaskTiGMScfeZwI5Dyl5x95rg6RygKDgeDzzq7pXuvgZYCYyOV2wiIqmookpjIvVdCbwUHPcENtR7rTQoExERwN0pr25FYyJHY2b/DtQAD8Xw3klmNt/M5peVlTV/cCIiIVRd69TWeasaEzksM7sCOA+4zN09KN4I9Kp3WlFQ9inuPsXdS9y9pLCwMK6xioiERRg3pIIEJxEzOxe4Dviqu5fXe+lZ4FIza2NmfYEBwDuJjE1EJMw+2ZAqI8mR/LO4RWNmjwDjgM5mVgr8kuhsrDbAdDMDmOPuV7v7B2b2OLCUaDfXNe5eG6/YRERSzScbUoXr9r64JRF3n3iY4nuPcv4NwA3xikdEJJUdaIm0+jERERFpPI2JiIhIzMI6JqIkIiKSAg6OiaglIiIijRXWgfVwRSMiIocVqYquGJWj7iwREWkszc4SEZGYRarrACURERGJwYExkTYZ4fraDlc0IiJyWJGqGnIy00lLs2SH8k+UREREUkAYN6QCJRERkZQQqaoL3XgIKImIiKSESHWNWiIiIhKbSFWtWiIiIhKbSLWSiIiIxChSXUe2urNERCQWkaoactUSERGRWGiKr4iIxCxSVRe6DakgjknEzO4zs61mtqReWSczm25mK4KfHYNyM7PbzWylmS02sxHxiktEJBVFqmrIbWUtkfuBcw8p+znwqrsPAF4NngN8ERgQPCYBd8YxLhGRlOLurW92lrvPBHYcUjwemBocTwXOr1f+gEfNATqYWfd4xSYikkqqauuoczQmAnR1903B8Waga3DcE9hQ77zSoExEpNWrqIouA9+qxkQ+i7s74I19n5lNMrP5Zja/rKws5ut/8PHumN8rIpJI5dXRXQ1b25jI4Ww50E0V/NwalG8EetU7rygo+xR3n+LuJe5eUlhYGFMQj8/fwHl3zGL60i0xvV9EJJHCuqshJD6JPAtcHhxfDjxTr/xbwSytk4Dd9bq9mt1XhvdgeM/2TH5kEUs2qkUiIuF2YEOqVtWdZWaPALOBQWZWamZXATcBZ5vZCuCs4DnAi8BqYCVwN/CDeMUF0cGpuy8voVNeFlfeP4+Pd0XieTkRkSY50BJpVd1Z7j7R3bu7e6a7F7n7ve6+3d3PdPcB7n6Wu+8IznV3v8bd+7n7MHefH6+4DuiSn819V4wiUlXLlffPY19lTbwvKSISkwMtEc3OCplB3fL502UjWLF1Hz98eCE1tXXJDklE5FM0JhJipw0s5Dfjh/L6h2X85vmlyQ5HRORTwjwmkpHsAMLg658rZu32/UyZuZreBXlceWrfZIckInJQmMdElEQCPz93MOu3l/ObF5bSrX02XxqmG+ZFJBwOjomEsCXS6ruzDkhLM/5w6QmMLO7ItY++y+xV25MdkogIoIH1lJGdmc49l5fQuyCXSQ/MZ+nHe5IdkogIkapazKBNRvi+ssMXUZJ1yM1i6pWjaZudweV/fYcNO8qTHZKItHKRqugKvmaW7FA+pUFJxMz6mVmb4HicmU02sw7xDS15enTIYeqVo6msruXy+95h+77KZIckIq1YWJeBh4a3RKYBtWbWH5hCdJ2rh+MWVQgM7JrPfVeMYuOuCFdOnU95lW5GFJHkiFTXhnJ6LzQ8idS5ew1wAXCHu/8MaPHTl0r6dOKPXx/B+6W7+N6DC6isqU12SCLSCkWqakM5vRcankSqzWwi0UUTnw/KMuMTUricPaQrN100nDdXbGPyI4t0V7uIJFykujaUM7Og4Unk28DJwA3uvsbM+gIPxi+scLmkpBe//MoQ/vHBFn725GLq6hq9DYqISMwiVeHtzmrQzYbuvhSYDGBmHYF8d785noGFzbfH9KW8qpbf/+NDcrPS+e35Q0M5U0JEWp5IdS2d8rKSHcZhNSiJmNkM4KvB+QuArWb2lrv/JI6xhc41Z/RnX2UNd85YRV6bDH7xxcFKJCISd5GqWnI6pHBLBGjv7nvM7DvAA+7+SzNbHM/Awuq6cwZRXlnDlJmrycvK4EdnDUh2SCLSwoV5im9Dk0hGsJ3tJcC/xzGe0DMzfvmV49hfVctt//cRbTLTuPr0fskOS0RasEhVLdkhHVhvaBL5NfAP4C13n2dmxwAr4hdWuKWlGTddOIzKmjpuemk5BnxPiURE4iRSXUtuKrdE3P0J4Il6z1cDF8UrqFSQkZ7GbZccj7tz40vLASUSEWl+7h7qKb4NHVgvAu4AxgRFbwI/cvfSeAWWCjLS0/jDhBMAlEhEJC4qa+pwD+eGVNDw+0T+CjwL9AgezwVlMTGzH5vZB2a2xMweMbNsM+trZnPNbKWZPWZm4ZzPdogDieS84d258aXl/OWNVckOSURakDBvSAUNTyKF7v5Xd68JHvcDhbFc0Mx6Er3npMTdhwLpwKXAzcBt7t4f2AlcFcvnJ4MSiYjES5g3pIKGJ5HtZvYNM0sPHt8AmrJrUwaQY2YZQC6wCfg88GTw+lTg/CZ8fsIdmkj++NoK3HVnu4g0TZg3pIKGz866kuiYyG2AA28DV8RyQXffaGa3AOuBCPAK0RsYdwWLPAKUAj1j+fxkOpBIstLTuOWVj9hXWcv15w7SDYkiErMD3VlhHRNp6OysdUTvWD/IzK4F/tDYCwbLpowH+gK7iM76OrcR758ETAIoLi5u7OXjLiM9jVu+djw5Wenc9cYqyqtq+NVXjiMtTYlERBrvQEsk1cdEDifWJU/OAta4e5m7VwNPEZ311SHo3gIoAjYe7s3uPsXdS9y9pLAwpmGZuEtLM357/lAmnXYMD8xex8+eXKzVf0UkJgdaImEdE2lod9bhxPqn9XrgJDPLJdqddSYwH3gduBh4lOiS8880IbakMzN+8cXBtG2Twa3TPyJSXcMfJpxIVgj3SBaR8DrQEglrd1ZTvtFiGjV297lEB9AXAu8HMUwBrgd+YmYrgQLg3ibEFgpmxuQzB/AfXz6WF9/fzKQHtUOiiDRO2Kf4HrUlYmZ7OXyyMCAn1ou6+y+BXx5SvBoYHetnhtl3xh5DXpsM/v3p9/n63XO574pRoV3WWUTCJeyzs47aEnH3fHdvd5hHvrs3pSus1Zk4upg/XzaSpZv2cPFdb1O6szzZIYlICgj7mIg66BPo3KHdePDK0ZTtreSiO99m+eY9yQ5JREKuJY+JSAw+d0wBT1x9MgBfu2s2c1c35Z5NEWnpIlW1pBm0CemknHBG1cIN7taOad8/hcL8Nnzzvnd46f1NyQ5JRELqwIZUYb1pWUkkSYo65jLt6lM4rkc7fvDwQqbMXKVlUkTkU8K8DDwoiSRVx7wsHvnuSXxpaHd+9+Jy/u3pJVTrpkQRqSdSFe4kohlWSZadmc4dE0+kuCCXO2esonRnOX+6bATtsjOTHZqIhECkKrz7q4NaIqGQlmZcf+5gbr5oGLNXbedrd87WFGARAT4ZEwkrJZEQmTCqmKlXjubj3RHO/9PbLFq/M9khiUiSRapqQzu9F5REQmdM/8489f1TyMlKY8KUOUxb0Kp3IBZp9SLVtaFd8gSUREJpQNd8nrnmVEYWd+SnT7zHb59fqlWARVopzc6SmHTKy+KBq0ZzxSl9uGfWGr59/zx2l1cnOywRSTB1Z0nMMtPT+NVXj+Pmi4YxZ/V2xv9pFiu27E12WCKSQOrOkiabMKqYRyedxL7KWi7489u6w12kFdEUX2kWI3t34rkfjqF/l7Z8/6GF3PCCxklEWjp31xRfaT7d2+fw2PdO4lsn9+buN9fw9XvmsnVPRbLDEpE4qaiO/qGYre4saS5tMtL59fih/GHCCbxfupsv3zGLd9bsSHZYIhIHB5aBz1VLRJrb+Sf25O/XjCG/TQYT757DlJmrqKvTAo4iLUnYdzUEJZGUNqhbPs/8yxi+MKQrv3txOVdNncf2fZXJDktEmsmBXQ01xfcQZtbBzJ40s+VmtszMTjazTmY23cxWBD87JiO2VJOfncmfLxvBb8Yfx1urtvOl299k9iptdCXSEhxIIrlZ4V0rN1ktkf8FXnb3wcDxwDLg58Cr7j4AeDV4Lg1gZnzz5D48/YNTyGuTwdfvmcOt0z/S7C2RFHewO0stkU+YWXvgNOBeAHevcvddwHhganDaVOD8RMeW6o7r0Z7n/uVULhpRxO2vruDr98xl0+5IssMSkRh9MiYS3pGHZETWFygD/mpmi8zsHjPLA7q6+4G76DYDXZMQW8rLa5PBLV87ntsmHM+Sjbs557aZPPvex8kOS0RiEKmqATQmcqgMYARwp7ufCOznkK4rj+4Te9ipRmY2yczmm9n8srKyuAebqi44sYgXJ4+lX5e2TH5kET96dBG7I1p7SySVHJziqzGRf1IKlLr73OD5k0STyhYz6w4Q/Nx6uDe7+xR3L3H3ksLCwoQEnKr6dM7jie+dzE/OHsjzizfxxT/M5O1V25Idlog0UKQqOq6pMZF63H0zsMHMBgVFZwJLgWeBy4Oyy4FnEh1bS5SRnsbkMwfw1PdPITszncvumcsNLyylIvgLR0TCKxUG1pPVRvoh8JCZZQGrgW8TTWiPm9lVwDrgkiTF1iId36sDz08+ld+9uIy731zD6x+W8fuLh3NisWZSi4TVgTGRMN9smJQk4u7vAiWHeenMRMfSmuRmZfDb84dx9pBu/HzaYi66822+O/YYfnz2wFAP3Im0VpHqWtLTjMx0S3YoRxTeeWMSN6cPLOSVH5/GhFHF/GXmar58+5ssWKf93EXCJlJVR05mOmZKIhIy+dmZ3HjhMB68ajQV1XVcfNfb/Pb5pZQHzWcRSb5IdU2ou7JASaTVGzugkJevHcvE0cXcM2sNX7htJjM+POzEOBFJsLBvSAVKIkK0VfK7C4bx2KSTyMpI44q/zmPyI4so26vFHEWSKewbUoGSiNTzuWMKeOlHY7n2rAG8vGQzZ936Bo/NW0/03k8RSbR128vpmJeZ7DCOSklE/kmbjHSuPWsgL/5oLIO65XP9tPeZ8Jc5LN+8J9mhibQqH3y8m+Wb9/LFod2THcpRKYnIYfXv0pZHv3sSN180jBVb9/Ll22fx6+eWsqdCS6eIJMK0BRvJTDe+enyPZIdyVEoickRpacaEUcW89tNxXDqqF399ew2fv+UNnl5Uqi4ukTiqrq3jmXc3cubgrnTMy0p2OEelJCKfqWNeFjdcMIxnrhlDz445/Pix95jwlzks/VhdXCLx8MaHZWzfX8VFI4uSHcpnUhKRBhte1IGnv38KN180jJVl+/jyHW/yi6cWaxaXSDObtrCUgrwsxg0K/yKzSiLSKAe6uF7/6TiuHNOXJ+aXcsYtM7hzxiot6ijSDHbur+LVZVsZf0JPMtPD/xUd/ggllNrnZvKf5w3hlR+fxknHFHDzy8s5+7Y3ePH9TRovEWmC5xZ/TFVtHReN7JnsUBpESUSa5JjCttxzeQkPfedz5GVl8IOHFnLRnW8zb+2OZIcmkpKmLShlcLd8juvRPtmhNIiSiDSLMf0788Lksdx04TA27orwtbtm852p81ixZW+yQxNJGSu37uW90t1cnAID6gcoiUizSU8zLh1dzIx/PYOfnTOIuat3cM4fZnLdk++xaXck2eGJhN6TCzaSnmaMPyE1urJASUTiICcrnWvO6M/M687g22P68vdFH3P672fwm+eXsm2fZnKJHE5tnfP0olLGDSykML9NssNpMCURiZuOeVn853lDePWnp/PV43vw17fWMPbm17n55eXsKq9KdngioTJr5Ta27KlMiXtD6lMSkbjr1SmXW752PNN/cjpnD+nKXW+sYuzNr3Pb9I+0jIpIYNqCUtrnZHLmsV2SHUqjKIlIwvQrbMvtE0/k5R+dxpj+nfnfV1dw6k2vcev0j9QykVZtT0U1//hgM189vgdtMsK99PuhkrLHOoCZpQPzgY3ufp6Z9QUeBQqABcA33V3fLC3QoG753PXNkSzZuJvbX13B7a+u4L5Za/jmyb35zql9KWibOv3BIo1VUV3L6rL9rCzbx8qt+1i1dR9LN+2hsqYu5bqyACxZN4aZ2U+AEqBdkEQeB55y90fN7C7gPXe/82ifUVJS4vPnz09EuBJHyzbt4Y+vr+TF9zeRnZHOZZ8r5runHUPXdtnJDk2k2bg7D7+znl8/t5TKmjoA0iza3du/sC1jB3Tm8lP6JGQ/dTNb4O4lzfJZyUgiZlYETAVuAH4CfAUoA7q5e42ZnQz8yt3POdrnKIm0LCu37uVPr6/imXc3kpGWxgUn9mTS6cfQr7BtskMTaZKK6lr+4+9LeHJBKWMHdOaSkl7079KWvp3zyE7CzoXNmUSS1Z31B+A6ID94XgDscvea4HkpkDoTpaVZ9O+Sz20TTuDaswZw95ureWJ+KY8v2MAXhnTle6f3Y0Rxx2SHKNJo67eXc/XfFrB00x5+dOYAJp85gPS0+Lc2EiXhScTMzgO2uvsCMxsXw/snAZMAiouLmzk6CYPeBXn89vxhXHvWQO5/ay0PzF7LPz7Ywui+nfju2GM4c3AX0lrQ/4TScr2+fCs/enQRAH+9YhRnDE6tmVcNkfDuLDO7EfgmUANkA+2Ap4FzUHeWHMa+yhoefWc9981aw8e7K+hTkMu3x/Tl4pFF5LVJ2twQkSNyd+54bSW3Tv+IId3bcdc3RlJckJvssA5K+TGRgxePtkT+NRhYfwKYVm9gfbG7//lo71cSaV2qa+t4eclm7p21hnc37CI/O4OJo4u5/JQ+9OyQk+zwRA76n1c+5I7XVnLhiT353YXDkjLucTQtNYkcQ3SKbydgEfANdz/qGhlKIq3XwvU7uXfWGl5eshl35+whXfnWyX04pV9BQma3iBzJn2es5L9f/pAJJb248cJhoex6bTFJpKmURKR0Zzl/m7Oex+atZ2d5Nf0K8/jWyX24cERP8rMzkx2etDL3v7WGXz23lPEn9ODWS04I7QC6kkhASUQOqKiu5YXFm3hgzjre27CLvKx0zj+xJxNHFzO0Z2rsyyCp7fF5G7hu2mK+MKQrf7psRKh3JVQSCSiJyOEsLt3FA7PX8dx7H1NZU8fwovZMHF3MV47vQVsNxEscPPPuRq597F3GDijk7m+NDP3SJUoiASUROZrd5dX8/d2NPDx3PR9u2UteVjpfPaEnE0b14vii9ho7kWbx6rItTHpwASN7d2Tqt0eTkxXuBAJKIgcpiUhDuDuLNuzikbnreW7xx1RU1zGgS1suHlnEBSN60iVfy6tIbPZWVHPGLTPo2i6bx753csq0dJVEAkoi0lh7Kqp5YfEmnpi/gYXrd5GeZowbWMjFI4v4/LFdQt8NIeFy44vLmPLmap695lSGFaXO2FtLWPZEJCnaZWcycXQxE0cXs6psH08uKOWphaW8unwr7bIz+PLwHlxwYk9KencM5dRMCY812/Zz31truHhEUUolkOamloi0ejW1dcxauY1n3v2Yl5dsJlJdS88OOZx/Yg/OP6EnA7rmf/aHSKvznanzmb1qG6//bFzKdYmqOyugJCLNbX9lDdOXbuHpRRt5c0UZdQ6Du+XzleN7cN7w7vQuyEt2iBICs1Zs4xv3zuW6cwfxg3H9kx1OoymJBJREJJ627q3gxcWbeH7xJuav2wnA8KL2nDe8O18a1p2ijuFZC0kSp6a2ji/d/iaR6lqm//j00C1p0hBKIgElEUmUjbsivLh4E88t/pjFpbuBaEI5d2g3vji0O307q4XSWjw4ey3/+cwH3PWNEZw7tHuyw4mJkkhASUSSYd32/by0ZDMvvb+J94KEMrhbPucO7cY5x3VjcLd83YPSQu0qr2LcLTM4tls7Hv7u51L296wkElASkWTbuCvCy0s28/KSaJeXO/TqlMNZx3bl7CFdGd2nExkhXv5CGudXz37AA7PX8sLksRzbvV2yw4mZkkhASUTCZOueCv5v2VamL93MW6u2U1VTR/ucTD4/uAufH9yF0wYW0j5Hi0Kmqg837+VLt7/JhFG9+N0Fw5IdTpMoiQSURCSs9lfWMPOjMqYv28Jry7eyq7ya9DSjpHfHg0mlf5e2Kdsd0tps3BXhkrtmU1Fdyys/Po2Ctm2SHVKTKIkElEQkFdTWOYvW7+S15Vt5bflWlm/eC0BRxxxOG1jIaQMKGdO/QEvXh1TZ3kom/GU2ZfsqeeS7J7WIVaGVRAJKIpKKPt4V4fUPtzLjwzLeXrmN/VW1pKcZI4s7ctrAzowdUMjQnu1DuxdFa7K7vJpL757D2m37efCq0ZT06ZTskJqFkkhASURSXVVNHQvX72TmR2XMXFHGko17AGiXncHJ/QoY078zY/p35pjOeer6SrD9lTV84965fLBxD/dcXsJpAwuTHVKzURIJKIlIS7NtXyVvrdzG2yu3M2vlNjbuigDQvX02Jx9TwEnBo1enHCWVOKqoruWqqfOYvWo7f74sde8HOZKUTiJm1gt4AOgKODDF3f/XzDoBjwF9gLXAJe6+82ifpSQiLZm7s35HObOCpDJn9Xa2768CokklmlA6MapPJ/qqpdJsauucq/+2gOlLt3DL147n4pFFyQ6p2aV6EukOdHf3hWaWDywAzgeuAHa4+01m9nOgo7tff7TPUhKR1sTdWVW2j9mrdzBn9Xbmrt7Otn3RpFKQl0VJn46M6hNNKkN6tAv19qxh9ti89Vw/7X3+67whXHlq32SHExcpvRS8u28CNgXHe81sGdATGA+MC06bCswAjppERFoTM6N/l3z6d8nnmyf1DpLKfuav3cE7a3cwf+1O/vHBFgByMtMZXtSeEb07MqK4IyOKO6T8tNREiFTVcuv0jzixuAPfHtMn2eGkhKTuJ2JmfYATgblA1yDBAGwm2t0lIkcQTSpt6d+lLZeOLgZgy54K5gUJZdH6ndw9czU1ddHehj4FuZzQqwPDijowvKg9x/VoR26WthSq77631rBlTyV//PoIdQ82UNL+BZlZW2AacK2776n/C3N3N7PD9rOZ2SRgEkBxcXEiQhVJGV3bZXPe8B6cN7wHEB0gXly6m4Xrd7Jw3U7mrN7B39/9GIA0g/5d2jKsZweO69GOY7u3Y0j3drTPbZ33q2zfV8mdM1Zx9pCujGohU3kTISlJxMwyiSaQh9z9qaB4i5l1d/dNwbjJ1sO9192nAFMgOiaSkIBWsDWKAAAKSElEQVRFUlR2Zjqj+3ZidN9PvhS37qng/Y27WVy6m/c37uaNj8qYtrD04Os9O+QECSWfwd3bMbhbPr0L8lr8fSt3vLaSSHUt1587ONmhpJSEJxGLNjnuBZa5+631XnoWuBy4Kfj5TKJjE2kNurTL5sx22Zx57Cc9xlv3VrD04z0s27SXZZv2sHTTHl5bvoWgJ4zszDQGdc1ncLd2DOyWz4AubRnQtS3d2mW3iG6ftdv287c567ikpBf9u7RNdjgpJRktkTHAN4H3zezdoOzfiCaPx83sKmAdcEkSYhNplbrkZ9NlUDbjBnU5WFZRXcuKLftYtnkPyzft5cMte5i+bAuPzd9w8Jy2bTLo16UtA7q0pV9hW/p2zqVv57b0LshNqc2afv/Kh2Smp/HjswYkO5SUk4zZWbOAI/3pcmYiYxGRI8vOTGdYUXuGFX2yVpS7s21fFSu37mNl2T5WbtnLyrJ9zPyojCcXfNIlZgY92ufQt3MevTrlUNQxl6KOORR1zKFnh1y65LchLSTdY+9u2MULizcx+fP96dIutfZKDwNNzRCRBjMzCvPbUJjfhpP7FfzTa3srqlm3vZzV2/azpmw/a7fvZ/W2/UxfuuXg/SwHZKWn0atTDr0L8uhdkEufgjyKg59FHXMSdo+Lu3Pji8soyMti0un9EnLNlkZJRESaRX52JkN7tj/sKreRqlo27ipnw84IpTsjlO4oZ932ctZu38/sVduJVNcePDcjzSjulEvfznn07ZxHn+BncadcenTIadYB/teWb2Xumh38ZvxxtG2jr8NY6L+aiMRdTlb6wRslD+XulO2rZN32ctZs28/abftZEzzeWrWNiuq6g+dmphtFHXMp7pRLn4JcenWKHhcX5NKrYy55n5EIqmvrWL5pLwvX72TBup3MXFFG3855B++zkcZTEhGRpDKz6MB+fvan7s+oq3M276lg7fb9rN9ezrod5cHP/Sxct5O9lTX/dH7ntll0b59DdmYaWRlpZKankZUePd62r5L3Nuw+2Orp2q4Np/Qr4Afj+muJmCZQEhGR0EpLM3p0yKFHhxxOOWTIwt3ZHalm/Y7yg48NO8rZtLuCyuo6Kqrr2BOpoaqmjqraOtplZzBhVC9G9O7IyN4d6dG+ZUxPTjYlERFJSWZGh9wsOuRmMbyoQ7LDabXUhhMRkZgpiYiISMyUREREJGZKIiIiEjMlERERiZmSiIiIxExJREREYqYkIiIiMTP31N0c0MzKiO490h7YXe+loz2vf9wZ2NZM4Rx6zaac39jXUq3+n3XukV5vaPmR6nvo85ZQ/88qO1L9m7PuR4oj1nObUv+w/9tvyPmJqP8gd//0QmaxcPeUfwBTGvr8kOP58YqhKec39rVUq/9nnXuk1xtafqT6tsT6f1bZkerfnHUPU/3D/m+/Jda/pXRnPdeI54e+Fq8YmnJ+Y19Ltfp/1rlHer2h5Uerb0ur/2eVtbb6h/3ffkPOT6n6p3R3VlOZ2Xx3L0l2HMmi+rfe+rfmuoPq35z1byktkVhNSXYASab6t16tue6g+jdb/Vt1S0RERJqmtbdERESkCZREREQkZkoiIiISMyWRIzCzNDO7wczuMLPLkx1PopnZODN708zuMrNxyY4n0cwsz8zmm9l5yY4l0czs2OD3/qSZfT/Z8SSamZ1vZneb2WNm9oVkx5NoZnaMmd1rZk825PwWmUTM7D4z22pmSw4pP9fMPjSzlWb288/4mPFAEVANlMYr1nhopvo7sA/IJoXq30x1B7geeDw+UcZPc9Tf3Ze5+9XAJcCYeMbb3Jqp/n939+8CVwMT4hlvc2um+q9296safM2WODvLzE4j+gX4gLsPDcrSgY+As4l+Kc4DJgLpwI2HfMSVwWOnu//FzJ5094sTFX9TNVP9t7l7nZl1BW5198sSFX9TNFPdjwcKiCbQbe7+fGKib7rmqL+7bzWzrwLfBx5094cTFX9TNVf9g/f9D/CQuy9MUPhN1sz1b9D3XkbzhR8e7j7TzPocUjwaWOnuqwHM7FFgvLvfCHyqy8LMSoGq4Glt/KJtfs1R/3p2Am3iEWc8NNPvfhyQBwwBImb2orvXxTPu5tJcv3t3fxZ41sxeAFImiTTT79+Am4CXUimBQLP/v98gLTKJHEFPYEO956XA545y/lPAHWY2FpgZz8ASpFH1N7MLgXOADsAf4xta3DWq7u7+7wBmdgVBiyyu0cVfY3/344ALif7x8GJcI0uMxv6//0PgLKC9mfV397viGVwCNPb3XwDcAJxoZr8Iks0RtaYk0ijuXg40uF+wpXH3p4gm0lbL3e9PdgzJ4O4zgBlJDiNp3P124PZkx5Es7r6d6HhQg7TIgfUj2Aj0qve8KChrLVpz/Vtz3UH1V/3jWP/WlETmAQPMrK+ZZQGXAs8mOaZEas31b811B9Vf9Y9j/VtkEjGzR4DZwCAzKzWzq9y9BvgX4B/AMuBxd/8gmXHGS2uuf2uuO6j+qn/i698ip/iKiEhitMiWiIiIJIaSiIiIxExJREREYqYkIiIiMVMSERGRmCmJiIhIzJREpEUys30Jvt49ZjYkwde81sxyE3lNkUPpPhFpkcxsn7u3bcbPywhu2kqYYDVZO9ICkGa2Fihx922JjEukPrVEpNUws0Izm2Zm84LHmKB8tJnNNrNFZva2mQ0Kyq8ws2fN7DXgVYvu9jjDojv+LTezh4IveoLykuB4n0V3xXzPzOYEe7JgZv2C5++b2W8P11oysz7B5kEPAEuAXmZ2p0V3WfzAzP5fcN5koAfwupm9HpR9IajHQjN7wsyaLYmKHJG766FHi3sA+w5T9jBwanBcDCwLjtsBGcHxWcC04PgKostmdwqejwN2E13ALo3o8hIHPm8G0VYBRHeF/Epw/N/AfwTHzwMTg+OrjxBjH6AOOKle2YHrpwfXGR48Xwt0Do47E92yIC94fj3wX8n+PejR8h9aCl5ak7OAIUHjAaBd8Nd6e2CqmQ0gmgAy671nurvvqPf8HXcvBTCzd4l+6c865DpVRBMGwAKiO8oBnAycHxw/DNxyhDjXufuces8vMbNJRLdu6E50s6zFh7znpKD8raB+WUSTnEhcKYlIa5JG9C/8ivqFZvZH4HV3vyDYFW5GvZf3H/IZlfWOazn8/0PV7u6fcc7RHLymmfUF/hUY5e47zex+otv2HsqIJryJjbyWSJNoTERak1eI7loHgJmdEBy255P9Fa6I4/XnABcFx5c28D3tiCaV3cHYyhfrvbYXyK/32WPMrD+AmeWZ2cCmhyxydEoi0lLlBkthH3j8BJgMlJjZYjNbyie7t/03cKOZLSK+rfNrgZ+Y2WKgP9HxlaNy9/eARcByol1gb9V7eQrwspm97u5lRBPgI8HnzwYGN2/4Ip+mKb4iCRLc0xFxdzezS4kOso9PdlwiTaExEZHEGQn8MZgWvAu4MsnxiDSZWiIiIhIzjYmIiEjMlERERCRmSiIiIhIzJREREYmZkoiIiMRMSURERGL2/wF0THjaEkokuwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_finder.plot(skip_end=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To restore the model and optimizer to their initial state use the `reset()` method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_finder.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch-lr-finder",
   "language": "python",
   "name": "torch-lr-finder"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
